#! /usr/bin/env python
# -*- coding: utf-8 -*-

import requests
import random
import time
import json
import pymysql

# http请求函数
# @param user_id 要采集的用户的id
# @param cursor_value 下一页的游标
# @param status_code 请求返回码
def x_request(user_id, tw_cursor, db_cursor):

    status_code = 0
    cursor_value = ''

    url = 'https://x.com/i/api/graphql/FptdvukWVyqC_Iez4vnrdQ/UserTweetsAndReplies?variables=%7B%22userId%22%3A%22'+user_id+'%22%2C%22count%22%3A20%2C%22cursor%22%3A%22'+tw_cursor+'%22%2C%22includePromotedContent%22%3Atrue%2C%22withCommunity%22%3Atrue%2C%22withVoice%22%3Atrue%7D&features=%7B%22rweb_video_screen_enabled%22%3Afalse%2C%22profile_label_improvements_pcf_label_in_post_enabled%22%3Atrue%2C%22rweb_tipjar_consumption_enabled%22%3Atrue%2C%22verified_phone_label_enabled%22%3Afalse%2C%22creator_subscriptions_tweet_preview_api_enabled%22%3Atrue%2C%22responsive_web_graphql_timeline_navigation_enabled%22%3Atrue%2C%22responsive_web_graphql_skip_user_profile_image_extensions_enabled%22%3Afalse%2C%22premium_content_api_read_enabled%22%3Afalse%2C%22communities_web_enable_tweet_community_results_fetch%22%3Atrue%2C%22c9s_tweet_anatomy_moderator_badge_enabled%22%3Atrue%2C%22responsive_web_grok_analyze_button_fetch_trends_enabled%22%3Afalse%2C%22responsive_web_grok_analyze_post_followups_enabled%22%3Atrue%2C%22responsive_web_jetfuel_frame%22%3Afalse%2C%22responsive_web_grok_share_attachment_enabled%22%3Atrue%2C%22articles_preview_enabled%22%3Atrue%2C%22responsive_web_edit_tweet_api_enabled%22%3Atrue%2C%22graphql_is_translatable_rweb_tweet_is_translatable_enabled%22%3Atrue%2C%22view_counts_everywhere_api_enabled%22%3Atrue%2C%22longform_notetweets_consumption_enabled%22%3Atrue%2C%22responsive_web_twitter_article_tweet_consumption_enabled%22%3Atrue%2C%22tweet_awards_web_tipping_enabled%22%3Afalse%2C%22responsive_web_grok_show_grok_translated_post%22%3Afalse%2C%22responsive_web_grok_analysis_button_from_backend%22%3Afalse%2C%22creator_subscriptions_quote_tweet_preview_enabled%22%3Afalse%2C%22freedom_of_speech_not_reach_fetch_enabled%22%3Atrue%2C%22standardized_nudges_misinfo%22%3Atrue%2C%22tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled%22%3Atrue%2C%22longform_notetweets_rich_text_read_enabled%22%3Atrue%2C%22longform_notetweets_inline_media_enabled%22%3Atrue%2C%22responsive_web_grok_image_annotation_enabled%22%3Atrue%2C%22responsive_web_enhance_cards_enabled%22%3Afalse%7D&fieldToggles=%7B%22withArticlePlainText%22%3Afalse%7D'
    params = {
    }

    # 换成自己的浏览器headers
    headers = {
        'key': 'value'
    }

    # 换成自己的cookies
    cookies = {
        'key': 'value'
    }

    # 请求数据为空的情况下（GET请求）就可以直接发送请求
    response = requests.get(url, headers=headers, cookies=cookies, params=params)
    status_code = response.status_code
    # 格式化为json格式
    try:
        result = response.json()
    except json.JSONDecodeError:
        # print('网络代码:'+str(response.status_code))
        return cursor_value, status_code
    
    # 提取instructions列表
    instructions = result.get("data", {})\
                   .get("user", {})\
                   .get("result", {})\
                   .get("timeline", {})\
                   .get("timeline", {})\
                   .get("instructions", [])
    # 过滤出 type 为 'TimelineAddEntries' 的项，并遍历获取贴文内容
    timeline_add_entries = [item for item in instructions if item.get("type") == "TimelineAddEntries"]
    if len(timeline_add_entries) <=0:
        # print('网络代码:'+str(response.status_code))
        return cursor_value, status_code

    timeline_add_entrie = timeline_add_entries[0].get("entries", {})
    for i, sub_entry in enumerate(timeline_add_entrie, start=1):
        timeline_add_entries_content = sub_entry.get("content", {})

        # 游标类型
        if timeline_add_entries_content.get("__typename", "") == 'TimelineTimelineCursor' and \
            timeline_add_entries_content.get("cursorType", "") == 'Bottom':
            cursor_value = timeline_add_entries_content.get("value", "")
        # 贴文内容
        else:
            # 检查 content 中是否有 "items" 数组，如果有代表有折叠回复，需要遍历该数组里的所有子项
            timeline_add_entries_content_items = timeline_add_entries_content.get("items")
            if timeline_add_entries_content_items and isinstance(timeline_add_entries_content_items, list):
                for j, timeline_add_entries_content_item in enumerate(timeline_add_entries_content_items, start=1):
                    timeline_add_entries_legacy = timeline_add_entries_content_item.get("item", {})\
                                                                                    .get("itemContent", {})\
                                                                                    .get("tweet_results", {})\
                                                                                    .get("result", {})\
                                                                                    .get("legacy", {})
                    # 只选本人的消息
                    if timeline_add_entries_legacy.get("id_str", "") != "" and timeline_add_entries_legacy.get("user_id_str", "")== user_id:
                        sql = "insert into tweets (time, twid, full_text, json_text, tw_cursor, favorite, reply, retweet) values\
                                (%(time)s, %(twid)s, %(full_text)s, %(json_text)s, %(tw_cursor)s, %(favorite)s, %(reply)s, %(retweet)s)"
                        effect_row = db_cursor.execute(sql, {'time': timeline_add_entries_legacy.get("created_at", "")\
                                                            , 'twid': timeline_add_entries_legacy.get("id_str", "")\
                                                            , 'full_text': timeline_add_entries_legacy.get("full_text", "")\
                                                            , 'json_text':str(timeline_add_entries_legacy)\
                                                            , 'tw_cursor':tw_cursor\
                                                            , 'favorite':timeline_add_entries_legacy.get("favorite_count", 0)\
                                                            , 'reply':timeline_add_entries_legacy.get("reply_count", 0)\
                                                            , 'retweet':timeline_add_entries_legacy.get("retweet_count", 0)})
                        if effect_row <=0:
                            print('数据库操作错误')
            else:
                timeline_add_entries_legacy = timeline_add_entries_content.get("itemContent", {})\
                                        .get("tweet_results", {})\
                                        .get("result", {})\
                                        .get("legacy", {})
                if timeline_add_entries_legacy.get("id_str", "") != "":
                    sql = "insert into tweets (time, twid, full_text, json_text, tw_cursor, favorite, reply, retweet) values\
                                (%(time)s, %(twid)s, %(full_text)s, %(json_text)s, %(tw_cursor)s, %(favorite)s, %(reply)s, %(retweet)s)"
                    effect_row = db_cursor.execute(sql, {'time': timeline_add_entries_legacy.get("created_at", "")\
                                                        , 'twid': timeline_add_entries_legacy.get("id_str", "")\
                                                        , 'full_text': timeline_add_entries_legacy.get("full_text", "")\
                                                        , 'json_text':str(timeline_add_entries_legacy)\
                                                        , 'tw_cursor':tw_cursor\
                                                        , 'favorite':timeline_add_entries_legacy.get("favorite_count", 0)\
                                                        , 'reply':timeline_add_entries_legacy.get("reply_count", 0)\
                                                        , 'retweet':timeline_add_entries_legacy.get("retweet_count", 0)})
                    if effect_row <=0:
                        print('数据库操作错误')

    return cursor_value, status_code



# * ------------- 程序开始 ---------------

tw_cursor = ''
# 这里要改成目标用户id
user_id = ''

# 连接mysql数据库
try:
    conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='root', db='tw', charset='utf8mb4')
    conn.autocommit(True)
    db_cursor = conn.cursor()
    print("数据库连接成功！")
except Exception as e:
    print("数据库连接失败，错误信息：", e)

n = 1
# 循环执行请求
while True:
    print('正在采集第 '+ str(n) + ' 页，cursor:'+tw_cursor)
    tw_cursor_t, status_code = x_request(user_id, tw_cursor, db_cursor)
    if tw_cursor_t == "" and status_code != 200:
        print("网络异常，代码"+str(status_code)+" ，将重新请求...")
        sleep_time = random.uniform(5, 6)
        time.sleep(sleep_time)
        continue
    elif tw_cursor_t == "" and status_code == 200:
        print('采集结束，程序退出')
        break
        
    tw_cursor = tw_cursor_t
    n += 1
    # 随机等待 1 到 3 秒
    sleep_time = random.uniform(3, 5)
    time.sleep(sleep_time)

    if n > 3:
        break

conn.commit()
# 关闭mysql连接
db_cursor.close()
conn.close()
